use strict;
use DBI;
use DBD::ODBC;
use POSIX;
use Time::HiRes qw(time);
use Storable qw(dclone);

my $flagLowDim = 0;
my $iter = 0;

#require 'ConvertUsingDummyVariables.pl';
require 'FilterData.pl';

my ($TRAINLD,$TESTLD,$TRAINHD,$TESTHD);

open (KEYINFO, '> NTDBNEW_Keys.txt');
open (LOG, '> NTDBNEW_Log.txt');
open ($TRAINHD, '> NTDBNEW_Train_HD.txt');
open ($TRAINLD, '> NTDBNEW_Train_LD.txt');
open ($TESTHD, '> NTDBNEW_Test_HD.txt');
open ($TESTLD, '> NTDBNEW_Test_LD.txt');
open (EDSYSBP, '> normalizedEDSYSBP.txt');
open (EDRESPRATE, '> normalizedEDRESPRATE.txt');
open (EDPULSE, '> normalizedEDPULSE.txt');

# Set connection parameters
my $dsn = 'dbi:ODBC:NTDBNEWDriver';
my ($username, $password) = ('trauma', 'trauma');
print LOG "username = $username, password = $password\n";
# Connect to the database
my $ntdb = DBI -> connect($dsn  ,$username,$password);

# Writing, printing, preparing and executing sql query
my $sql = "EXEC sp_NTDBNEW_ExtractRelevantColsFromAllTables14_BODYREGION_AISSCORE";
print LOG "SQL COMMAND EXECUTED: $sql\n";
my $sth = $ntdb -> prepare($sql);
$sth -> execute();

# Declare hashes
my (%hAISCODE, %hAISSCORE, %hBODYREGION, %hINC_KEY, %hAGE, %hGENDER, %hDCODE, %hLOS, %hECODE, %hECODE_ROW, %hECODE_COL, %hEDSYSBP, %hEDRESPRATE, %hEDPULSE, %hEDGCSEYE, %hEDGCSVERB, %hEDGCSMOTOR, %hEDGCSTOTAL, %hINJURYTYPE, %hINJURYSITE, %hYOINJ, %hOUTPUT);
my (%hINC_KEY_AISCODE, %hINC_KEY_AISSCORE, %hINC_KEY_BODYREGION, %hINC_KEY_BODYREGION_AISSCORE, %hINC_KEY_DCODE, %hINC_KEY_ECODE, %hINC_KEY_ECODE_ROW, %hINC_KEY_ECODE_COL, %hINC_KEY_EDGCSEYE, %hINC_KEY_EDGCSVERB, %hINC_KEY_EDGCSMOTOR, %hINC_KEY_EDGCSTOTAL, %hINC_KEY_INJURYTYPE, %hINC_KEY_INJURYSITE, %hDCODE_DCODE, %hDCODE_INJURYTYPE, %hAISCODE_AISCODE, %hAISCODE_INJURYTYPE, %hBODYREGION_AISSCORE, %hBODYREGIONAISSCORE_BODYREGIONAISSCORE);

# Declare corresponding hash keys
my ($INC_KEY, $AISCODE, $AISSCORE, $BODYREGION, $AGE, $GENDER, $DCODE, $LOS, $DISCHDISP, $ECODE1, $ECODE2, $EDSYSBP, $EDRESPRATE, $EDPULSE, $EDGCSEYE, $EDGCSVERB, $EDGCSMOTOR, $EDGCSTOTAL, $EDDISP, $INJURYTYPE, $INJURYSITE, $YOINJ);

# Setting subroutine for converting to dummy variables

my ($EDGCSEYE_DUMMY, $EDGCSVERB_DUMMY, $EDGCSMOTOR_DUMMY, $EDGCSTOTAL_DUMMY);
my $OUTPUT;

my $startTime = time();
while(($INC_KEY, $AISCODE, $AISSCORE, $AGE, $GENDER, $DCODE, $LOS, $DISCHDISP, $ECODE1, $ECODE2, $EDSYSBP, $EDRESPRATE, $EDPULSE, $EDGCSEYE, $EDGCSVERB, $EDGCSMOTOR, $EDGCSTOTAL, $INJURYTYPE, $EDDISP, $INJURYSITE, $YOINJ) = $sth -> fetchrow_array())
{
    $iter++;
    
    $INC_KEY = trim($INC_KEY); $AISCODE = trim($AISCODE); $AISSCORE = trim($AISSCORE);
    
    $BODYREGION = getBodyregionFromAISCODE($AISCODE);
    

    #$AGE = trim($AGE); $GENDER = trim($GENDER); 
    $DCODE = trim($DCODE);
    
    # Add 1 to LOS to handle patients with LOS = 0 (Dead on arrival).
    # Since, the survival function S(t=0) = 1. Therefore, theoretically,
    # a person cannot be dead when the study was started.
    $LOS = trim($LOS) + 1;
    
    $ECODE1 = trimIfEmpty($ECODE1); $ECODE2 = trimIfEmpty($ECODE2);
    #$EDSYSBP = trim($EDSYSBP); $EDRESPRATE = trim($EDRESPRATE); $EDPULSE = trim($EDPULSE); $EDGCSEYE = trimIfEmpty($EDGCSEYE); $EDGCSVERB = trimIfEmpty($EDGCSVERB); $EDGCSMOTOR = trimIfEmpty($EDGCSMOTOR); $EDGCSTOTAL = trimIfEmpty($EDGCSTOTAL);
    $EDDISP = trimIfEmpty($EDDISP);
    #$INJURYTYPE = trim($INJURYTYPE); $INJURYSITE = trim($INJURYSITE);
    $YOINJ = trimIfEmpty($YOINJ);
    
    $DISCHDISP = trimIfEmpty($DISCHDISP);
    $OUTPUT = filterStatusNEW($DISCHDISP, $EDDISP);

    
    # ICD9 Filtering
    $DCODE = filterICD9($DCODE);
    
    # EDGCS Filtering
    #($EDGCSEYE, $EDGCSVERB, $EDGCSMOTOR, $EDGCSTOTAL) = filterGCS($EDGCSEYE, $EDGCSVERB, $EDGCSMOTOR, $EDGCSTOTAL);
    
    # ECODE Filtering
    my ($ECODE1_ROW, $ECODE1_COL);
    ($ECODE1,$ECODE1_ROW,$ECODE1_COL) = filterECODE($ECODE1);

    my ($ECODE2_ROW, $ECODE2_COL);
    ($ECODE2,$ECODE2_ROW,$ECODE2_COL) = filterECODE($ECODE2);

    # EDSYSBP Normalization and Fitting
    #$EDSYSBP = normalizeEDSYSBP($EDSYSBP,$AGE);
    
    # EDRESPRATE Normalization and Fitting
    #$EDRESPRATE = normalizeEDRESPRATE($EDRESPRATE,$AGE);

    #TODO: $EDPULSE HERE
    #$EDPULSE = normalizeEDPULSE($EDPULSE,$AGE);
    
    if($OUTPUT != -1)
    {
        if($DCODE eq "REMOVEPATIENT" or $ECODE1_ROW eq "REMOVEPATIENT" or $ECODE1_ROW eq "REMOVEPATIENT" or $ECODE2_ROW eq "REMOVEPATIENT")
        {
            $hINC_KEY{$INC_KEY} = -1;
        }

        my $flagRemovePatient = 0;
        if(exists $hINC_KEY{$INC_KEY})
        {
            if($hINC_KEY{$INC_KEY} == -1)
            {
                $flagRemovePatient = 1;
            }
        }
        if($flagRemovePatient == 0)
        {
            # Start filling hashes
            $hOUTPUT{$INC_KEY} = $OUTPUT;
            $hINC_KEY{$INC_KEY} = 1;                            # key = INC_KEY, value = 1

            $hINC_KEY_AISCODE{$INC_KEY}{$AISCODE} = 1;          # key1 = INC_KEY, key2 = AISCODE, value = 1
            $hINC_KEY_AISSCORE{$INC_KEY}{$AISSCORE} = 1;        # key1 = INC_KEY, key2 = AISSCORE, value = 1
            
            # Populate BODYREGION_AISSCORE combined keys. Keep only the worst injury for a given BODYREGION (the one with highest AISSCORE)
            if(!exists($hINC_KEY_BODYREGION{$INC_KEY}{$BODYREGION}))
            {
                $hINC_KEY_BODYREGION{$INC_KEY}{$BODYREGION} = 1;
                $hINC_KEY_BODYREGION_AISSCORE{$INC_KEY}{$BODYREGION}{$AISSCORE} = 1;
            }
            else
            {
                foreach my $score (sort keys %{$hINC_KEY_BODYREGION_AISSCORE{$INC_KEY}{$BODYREGION}})
                {
                    if($AISSCORE > $score)
                    {
                        delete($hINC_KEY_BODYREGION_AISSCORE{$INC_KEY}{$BODYREGION});
                        $hINC_KEY_BODYREGION_AISSCORE{$INC_KEY}{$BODYREGION}{$AISSCORE} = 1;
                    }
                }
            }
            
            #$hAGE{$INC_KEY} = $AGE;                             # key = INC_KEY, value = age of the person
            #if($GENDER eq "Male")
            #{
            #    $hGENDER{$INC_KEY} = 1;                         # key = INC_KEY, value = gender of the person
            #}
            #else
            #{
            #    $hGENDER{$INC_KEY} = 0;
            #}
            $hINC_KEY_DCODE{$INC_KEY}{$DCODE} = 1;              # key1 = INC_KEY, key2 = DCODE, value = 1
            $hLOS{$INC_KEY} = $LOS;                             # key = INC_KEY, value = length of stay

            if($ECODE1_ROW ne "REMOVEPATIENT" and $ECODE1_ROW ne "ECODENOTFOUND")
            {
                $hINC_KEY_ECODE{$INC_KEY}{$ECODE1} = 1;
                $hINC_KEY_ECODE_ROW{$INC_KEY}{$ECODE1_ROW} = 1;
                $hINC_KEY_ECODE_COL{$INC_KEY}{$ECODE1_COL} = 1;
            }
            if($ECODE2_ROW ne "REMOVEPATIENT" and $ECODE2_ROW ne "ECODENOTFOUND")
            {
                $hINC_KEY_ECODE{$INC_KEY}{$ECODE2} = 1;
                $hINC_KEY_ECODE_ROW{$INC_KEY}{$ECODE2_ROW} = 1;
                $hINC_KEY_ECODE_COL{$INC_KEY}{$ECODE2_COL} = 1;
            }
            
            #$hEDSYSBP{$INC_KEY} = $EDSYSBP;                     # key = INC_KEY, value = systolic blood pressure in ED
            #$hEDRESPRATE{$INC_KEY} = $EDRESPRATE;               # key = INC_KEY, value = respiratory rate in ED
            #$hEDPULSE{$INC_KEY} = $EDPULSE;
            #$hINC_KEY_EDGCSEYE{$INC_KEY}{$EDGCSEYE} = 1;
            #$hINC_KEY_EDGCSVERB{$INC_KEY}{$EDGCSVERB} = 1;
            #$hINC_KEY_EDGCSMOTOR{$INC_KEY}{$EDGCSMOTOR} = 1;
            #$hINC_KEY_EDGCSTOTAL{$INC_KEY}{$EDGCSTOTAL} = 1;
            #$hINC_KEY_INJURYTYPE{$INC_KEY}{$INJURYTYPE} = 1;
            #$hINC_KEY_INJURYSITE{$INC_KEY}{$INJURYSITE} = 1;
            
            if($YOINJ == 2010)
            {
                $hYOINJ{$INC_KEY} = 1;                      #Test data
            }
            else
            {
                $hYOINJ{$INC_KEY} = 0;                      #Training data
            }
        }
    }
    print LOG "Processing row $iter...$INC_KEY, " . keys(%{$hINC_KEY_ECODE{$INC_KEY}}) . "\n";    
}
my $runTime = time() - $startTime;
print LOG "Processed $iter rows of SQL data.\n";
print LOG "Time taken = $runTime seconds.\n\n";

foreach my $key (sort keys(%hINC_KEY))
{
    if($hINC_KEY{$key} == -1 or keys(%{$hINC_KEY_ECODE{$key}}) == 0 or keys(%{$hINC_KEY_DCODE{$key}}) == 0) # if REMOVEPATIENT
    {
       delete($hINC_KEY{$key});
       delete($hAGE{$key});
       delete($hGENDER{$key});
       delete($hLOS{$key});
       delete($hEDSYSBP{$key});
       delete($hEDRESPRATE{$key});
       delete($hEDPULSE{$key});
       delete($hYOINJ{$key});
       delete($hOUTPUT{$key});
       
       delete($hINC_KEY_AISCODE{$key});
       delete($hINC_KEY_AISSCORE{$key});
       delete($hINC_KEY_BODYREGION{$key});
       delete($hINC_KEY_DCODE{$key});
       delete($hINC_KEY_ECODE{$key});
       delete($hINC_KEY_ECODE_ROW{$key});
       delete($hINC_KEY_ECODE_COL{$key});
       delete($hINC_KEY_EDGCSEYE{$key});
       delete($hINC_KEY_EDGCSVERB{$key});
       delete($hINC_KEY_EDGCSMOTOR{$key});
       delete($hINC_KEY_EDGCSTOTAL{$key});
       delete($hINC_KEY_INJURYTYPE{$key});
       delete($hINC_KEY_INJURYSITE{$key});
       delete($hDCODE_DCODE{$key});
       delete($hDCODE_INJURYTYPE{$key});
       delete($hAISCODE_AISCODE{$key});
       delete($hAISCODE_INJURYTYPE{$key});
       delete($hBODYREGION_AISSCORE{$key});
       delete($hINC_KEY_BODYREGION_AISSCORE{$key});
       delete($hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$key});
    }
    else
    {
        foreach my $i (sort keys %{$hINC_KEY_DCODE{$key}})
        {
            if(exists($hDCODE{$i}))
            {
                $hDCODE{$i} = $hDCODE{$i} + 1;
            }
            else
            {
                $hDCODE{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_AISCODE{$key}})
        {
            if(exists($hAISCODE{$i}))
            {
                $hAISCODE{$i} = $hAISCODE{$i} + 1;
            }
            else
            {
                $hAISCODE{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_AISSCORE{$key}})
        {
            if(exists($hAISSCORE{$i}))
            {
                $hAISSCORE{$i} = $hAISSCORE{$i} + 1;
            }
            else
            {
                $hAISSCORE{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_BODYREGION{$key}})
        {
            if(exists($hBODYREGION{$i}))
            {
                $hBODYREGION{$i} = $hBODYREGION{$i} + 1;
            }
            else
            {
                $hBODYREGION{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_ECODE{$key}})
        {
            if(exists($hECODE{$i}))
            {
                $hECODE{$i} = $hECODE{$i} + 1;
            }
            else
            {
                $hECODE{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_ECODE_ROW{$key}})
        {
            if(exists($hECODE_ROW{$i}))
            {
                $hECODE_ROW{$i} = $hECODE_ROW{$i} + 1;
            }
            else
            {
                $hECODE_ROW{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_ECODE_COL{$key}})
        {
            if(exists($hECODE_COL{$i}))
            {
                $hECODE_COL{$i} = $hECODE_COL{$i} + 1;
            }
            else
            {
                $hECODE_COL{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_EDGCSEYE{$key}})
        {
            $hEDGCSEYE{$i} = 1;
        }
        foreach my $i (sort keys %{$hINC_KEY_EDGCSVERB{$key}})
        {
            $hEDGCSVERB{$i} = 1;
        }
        foreach my $i (sort keys %{$hINC_KEY_EDGCSMOTOR{$key}})
        {
            $hEDGCSMOTOR{$i} = 1;
        }
        foreach my $i (sort keys %{$hINC_KEY_EDGCSTOTAL{$key}})
        {
            $hEDGCSTOTAL{$i} = 1;
        }
        foreach my $i (sort keys %{$hINC_KEY_INJURYTYPE{$key}})
        {
            if(exists($hINJURYTYPE{$i}))
            {
                $hINJURYTYPE{$i} = $hINJURYTYPE{$i} + 1;
            }
            else
            {
                $hINJURYTYPE{$i} = 1;
            }
        }
        foreach my $i (sort keys %{$hINC_KEY_INJURYSITE{$key}})
        {
            if(exists($hINJURYSITE{$i}))
            {
                $hINJURYSITE{$i} = $hINJURYSITE{$i} + 1;
            }
            else
            {
                $hINJURYSITE{$i} = 1;
            }
        }
    }
}
print LOG "Number of INC_KEY keys:\nhOUTPUT = " . (keys %hOUTPUT) . "\nhINC_KEY = " . (keys %hINC_KEY) . "\nhINC_KEY_AISCODE = " . (keys %hINC_KEY_AISCODE) . "\nhINC_KEY_AISSCORE = " . (keys %hINC_KEY_AISSCORE) . "\nhINC_KEY_BODYREGION = " . (keys %hINC_KEY_BODYREGION) . "\nhAGE = " . (keys %hAGE) . "\nhGENDER = " . (keys %hGENDER) . "\nhINC_KEY_DCODE = " . (keys %hINC_KEY_DCODE) . "\nhLOS = " . (keys %hLOS) . "\nhINC_KEY_ECODE = " . (keys %hINC_KEY_ECODE) . "\nhINC_KEY_ECODE_ROW = " . (keys %hINC_KEY_ECODE_ROW) . "\nhINC_KEY_ECODE_COL = " . (keys %hINC_KEY_ECODE_COL) . "\nhEDSYSBP = " . (keys %hEDSYSBP) . "\nhEDRESPRATE = " . (keys %hEDRESPRATE) . "\nhINC_KEY_EDGCSEYE = " . (keys %hINC_KEY_EDGCSEYE) . "\nhINC_KEY_EDGCSVERB = " . (keys %hINC_KEY_EDGCSVERB) . "\nhINC_KEY_EDGCSMOTOR = " . (keys %hINC_KEY_EDGCSMOTOR) . "\nhINC_KEY_EDGCSTOTAL = " . (keys %hINC_KEY_EDGCSTOTAL) . "\nhINC_KEY_INJURYTYPE = " . (keys %hINC_KEY_INJURYTYPE) . "\nhINC_KEY_INJURYSITE = " . (keys %hINC_KEY_INJURYSITE) . "\n\n";
# Printing some counts
print LOG "Counts:\n------------------------------------------\n";
print LOG "AISSCORE\n";
foreach my $i (sort keys %hAISSCORE)
{
    print LOG "$i $hAISSCORE{$i}\n";
}
print LOG "\nBODYREGION\n";
foreach my $i (sort keys %hBODYREGION)
{
    print LOG "$i $hBODYREGION{$i}\n";
}
print LOG "\nECODE_ROW\n";
foreach my $i (sort keys %hECODE_ROW)
{
    print LOG "$i $hECODE_ROW{$i}\n";
}
print LOG "\nECODE_COL\n";
foreach my $i (sort keys %hECODE_COL)
{
    print LOG "$i $hECODE_COL{$i}\n";
}
print LOG "\nINJURYTYPE\n";
foreach my $i (sort keys %hINJURYTYPE)
{
    print LOG "$i $hINJURYTYPE{$i}\n";
}
print LOG "\nINJURYSITE\n";
foreach my $i (sort keys %hINJURYSITE)
{
    print LOG "$i $hINJURYSITE{$i}\n";
}
print LOG "\nICD9 Codes\n";
foreach my $i (sort keys %hDCODE)
{
    print LOG "$i $hDCODE{$i}\n";
}
print LOG "\nECODES\n";
foreach my $i (sort keys %hECODE)
{
    print LOG "$i $hECODE{$i}\n";
}
print LOG "\nAISCODES\n";
foreach my $i (sort keys %hAISCODE)
{
    print LOG "$i $hAISCODE{$i}\n";
}


# Note: 1=age, 2=gender, 3=sysbp, 4=resprate, 5=pulse
print KEYINFO "seq=0: Intercept\n";
print KEYINFO "seq=1: Age\n";
print KEYINFO "seq=2: Gender\n";
print KEYINFO "seq=3: Systolic BP\n";
print KEYINFO "seq=4: Resp Rate\n";
print KEYINFO "seq=5: Pulse\n\n";

# assign sequence number for ICD9 codes.
my $seqNum = 6;		
foreach my $i (sort keys %hDCODE)
{
    $hDCODE{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "ICD9 Code Keys... ". (keys %hDCODE) ." keys\n";
foreach my $i (sort keys %hDCODE)
{
    print KEYINFO "seq=$hDCODE{$i}: ICD9=$i\n";
}
print KEYINFO "\n";

# assign sequence number for AISCODE
foreach my $i (sort keys %hAISCODE)
{
    $hAISCODE{$i}=$seqNum;
    $seqNum++;        
}
print KEYINFO "AISCODE Keys... ". (keys %hAISCODE) ." keys\n";
foreach my $i (sort keys %hAISCODE)
{
    print KEYINFO "seq=$hAISCODE{$i}: AISCODE=$i\n";
}
print KEYINFO "\n";

## assign sequence number for AISSCORE 
#foreach my $i (sort keys %hAISSCORE)
#{
#    $hAISSCORE{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "AISSCORE Keys... ". (keys %hAISSCORE) ." keys\n";
#foreach my $i (sort keys %hAISSCORE)
#{
#    print KEYINFO "seq=$hAISSCORE{$i}, AISSCORE=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for BODYREGION
#foreach my $i (sort keys %hBODYREGION)
#{
#    $hBODYREGION{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "BODYREGION Keys... ". (keys %hBODYREGION) ." keys\n";
#foreach my $i (sort keys %hBODYREGION)
#{
#    print KEYINFO "seq=$hBODYREGION{$i}, BODYREGION=$i\n";
#}
#print KEYINFO "\n";

## assign sequence number for ECODE
#foreach my $i (sort keys %hECODE)
#{
#    $hECODE{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "ECODE Keys... ". (keys %hECODE) ." keys\n";
#foreach my $i (sort keys %hECODE)
#{
#    print KEYINFO "seq=$hECODE{$i}: ECODE=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for ECODE_ROW
#foreach my $i (sort keys %hECODE_ROW)
#{
#    $hECODE_ROW{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "ECODE_ROW Keys... ". (keys %hECODE_ROW) ." keys\n";
#foreach my $i (sort keys %hECODE_ROW)
#{
#    print KEYINFO "seq=$hECODE_ROW{$i}: ECODE_ROW=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for ECODE_COL
#foreach my $i (sort keys %hECODE_COL)
#{
#    $hECODE_COL{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "ECODE_COL Keys... ". (keys %hECODE_COL) ." keys\n";
#foreach my $i (sort keys %hECODE_COL)
#{
#    print KEYINFO "seq=$hECODE_COL{$i}: ECODE_COL=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for EDGCSEYE
#foreach my $i (sort keys %hEDGCSEYE)
#{
#    $hEDGCSEYE{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "EDGCSEYE Keys... ". (keys %hEDGCSEYE) ." keys\n";
#foreach my $i (sort keys %hEDGCSEYE)
#{
#    print KEYINFO "seq=$hEDGCSEYE{$i}: EDGCSEYE=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for EDGCSVERB
#foreach my $i (sort keys %hEDGCSVERB)
#{
#    $hEDGCSVERB{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "EDGCSVERB Keys... ". (keys %hEDGCSVERB) ." keys\n";
#foreach my $i (sort keys %hEDGCSVERB)
#{
#    print KEYINFO "seq=$hEDGCSVERB{$i}: EDGCSVERB=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for EDGCSMOTOR
#foreach my $i (sort keys %hEDGCSMOTOR)
#{
#    $hEDGCSMOTOR{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "EDGCSMOTOR Keys...". (keys %hEDGCSMOTOR) ." keys\n";
#foreach my $i (sort keys %hEDGCSMOTOR)
#{
#    print KEYINFO "seq=$hEDGCSMOTOR{$i}: EDGCSMOTOR=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for EDGCSTOTAL
#foreach my $i (sort keys %hEDGCSTOTAL)
#{
#    $hEDGCSTOTAL{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "EDGCSTOTAL Keys...". (keys %hEDGCSTOTAL) ." keys\n";
#foreach my $i (sort keys %hEDGCSTOTAL)
#{
#    print KEYINFO "seq=$hEDGCSTOTAL{$i}: EDGCSTOTAL=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for INJURYTYPE
#foreach my $i (sort keys %hINJURYTYPE)
#{
#    $hINJURYTYPE{$i}=$seqNum;
#    $seqNum++;        
#}
#print KEYINFO "INJURYTYPE Keys... ". (keys %hINJURYTYPE) ." keys\n";
#foreach my $i (sort keys %hINJURYTYPE)
#{
#    print KEYINFO "seq=$hINJURYTYPE{$i}: INJURYTYPE=$i\n";
#}
#print KEYINFO "\n";
#
## assign sequence number for INJURYSITE
#foreach my $i (sort keys %hINJURYSITE)
#{
#    $hINJURYSITE{$i}=$seqNum;
#    $seqNum++;
#}
#print KEYINFO "INJURYSITE Keys... ". (keys %hINJURYSITE) ." keys\n";
#foreach my $i (sort keys %hINJURYSITE)
#{
#    print KEYINFO "seq=$hINJURYSITE{$i}: INJURYSITE=$i\n";
#}
#print KEYINFO "\n";

#my faster implementation
my $count = 0;
my %hTMP;

$startTime = time();
foreach my $key (sort keys %hINC_KEY_DCODE)
{
    %hTMP = %{dclone(\%{$hINC_KEY_DCODE{$key}})};
    foreach my $code1 (sort keys %hTMP)
    {
        delete($hTMP{$code1});  # to avoid [code1,code1] combinations
        foreach my $code2 (sort keys %hTMP)
        {
            if(!exists($hDCODE_DCODE{$code1}{$code2}))
            {
                $count++;
                print LOG "Processing two-way [ICD9,ICD9] interactions...$count\n";
                $hDCODE_DCODE{$code1}{$code2} = 1;
            }
        }
    }
}
foreach my $code1 (sort keys %hDCODE_DCODE)
{
    foreach my $code2 (sort keys %{$hDCODE_DCODE{$code1}})
    {
        $hDCODE_DCODE{$code1}{$code2} = $seqNum;
        $seqNum++;
    }
}
print KEYINFO "Two-way [ICD9,ICD9] Keys... $count keys\n";
foreach my $i (sort keys %hDCODE_DCODE)
{
    foreach my $j (sort keys %{$hDCODE_DCODE{$i}})
    {
        print KEYINFO "seq=$hDCODE_DCODE{$i}{$j}: [ICD9,ICD9]=$i, $j\n";
    }
}
print KEYINFO "\n";
$runTime = time() - $startTime;
print LOG "Processed $count two-way [ICD9,ICD9] interactions.\n";
print LOG "Time taken = $runTime seconds.\n\n";

## assign sequence number for [ICD9,INJURYTYPE] two-way interactions
#$count = 0;
#$startTime = time();
#foreach my $key (sort keys %hINC_KEY_DCODE)
#{
#    foreach my $code (sort keys %{$hINC_KEY_DCODE{$key}})
#    {
#        foreach my $injuryType (sort keys %{$hINC_KEY_INJURYTYPE{$key}})
#        {
#            if(!exists($hDCODE_INJURYTYPE{$code}{$injuryType}))
#            {
#                $count++;
#                print LOG "Processing two-way [ICD9,INJURYTYPE] interactions...$count\n";
#                $hDCODE_INJURYTYPE{$code}{$injuryType} = 1;
#            }
#        }
#    }
#}
#foreach my $code (sort keys %hDCODE_INJURYTYPE)
#{
#    foreach my $injuryType (sort keys %{$hDCODE_INJURYTYPE{$code}})
#    {
#        $hDCODE_INJURYTYPE{$code}{$injuryType} = $seqNum;
#        $seqNum++;
#    }
#}
#print KEYINFO "Two-way [ICD9,INJURYTYPE] Keys... $count keys\n";
#foreach my $code (sort keys %hDCODE_INJURYTYPE)
#{
#    foreach my $injuryType (sort keys %{$hDCODE_INJURYTYPE{$code}})
#    {
#        print KEYINFO "seq=$hDCODE_INJURYTYPE{$code}{$injuryType}: [ICD9,INJURYTYPE]=$code, $injuryType\n";
#    }
#}
#print KEYINFO "\n";
#$runTime = time() - $startTime;
#print LOG "Processed $count two-way [ICD9,INJURYTYPE] interactions.\n";
#print LOG "Time taken = $runTime seconds.\n\n";

# assign sequence number for [AISCODE,AISCODE] two-way interactions
$count = 0;
$startTime = time();
foreach my $key (sort keys %hINC_KEY_AISCODE)
{
    %hTMP = %{dclone(\%{$hINC_KEY_AISCODE{$key}})};
    foreach my $code1 (sort keys %hTMP)
    {
        delete($hTMP{$code1});  # to avoid [code1,code1] combinations
        foreach my $code2 (sort keys %hTMP)
        {
            if(!exists($hAISCODE_AISCODE{$code1}{$code2}))
            {
                $count++;
                print LOG "Processing two-way [AISCODE,AISCODE] interactions...$count\n";
                $hAISCODE_AISCODE{$code1}{$code2} = 1;
            }
        }
    }
}
foreach my $code1 (sort keys %hAISCODE_AISCODE)
{
    foreach my $code2 (sort keys %{$hAISCODE_AISCODE{$code1}})
    {
        $hAISCODE_AISCODE{$code1}{$code2} = $seqNum;
        $seqNum++;
    }
}
print KEYINFO "Two-way [AISCODE,AISCODE] Keys... $count keys\n";
foreach my $code1 (sort keys %hAISCODE_AISCODE)
{
    foreach my $code2 (sort keys %{$hAISCODE_AISCODE{$code1}})
    {
        print KEYINFO "seq=$hAISCODE_AISCODE{$code1}{$code2}: [AISCODE,AISCODE]=$code1, $code2\n";
    }
}
print KEYINFO "\n";
$runTime = time() - $startTime;
print LOG "Processed $count two-way [AISCODE,AISCODE] interactions.\n";
print LOG "Time taken = $runTime seconds.\n\n";

## assign sequence number for [AISCODE,INJURYTYPE] two-way interactions
#$count = 0;
#$startTime = time();
#foreach my $key (sort keys %hINC_KEY_AISCODE)
#{
#    foreach my $code (sort keys %{$hINC_KEY_AISCODE{$key}})
#    {
#        foreach my $injuryType (sort keys %{$hINC_KEY_INJURYTYPE{$key}})
#        {
#            if(!exists($hAISCODE_INJURYTYPE{$code}{$injuryType}))
#            {
#                $count++;
#                print LOG "Processing two-way [AISCODE,INJURYTYPE] interactions...$count\n";
#                $hAISCODE_INJURYTYPE{$code}{$injuryType} = 1;
#            }
#        }
#    }
#}
#foreach my $code (sort keys %hAISCODE_INJURYTYPE)
#{
#    foreach my $injuryType (sort keys %{$hAISCODE_INJURYTYPE{$code}})
#    {
#        $hAISCODE_INJURYTYPE{$code}{$injuryType} = $seqNum;
#        $seqNum++;
#    }
#}
#print KEYINFO "Two-way [AISCODE,INJURYTYPE] Keys... $count keys\n";
#foreach my $code (sort keys %hAISCODE_INJURYTYPE)
#{
#    foreach my $injuryType (sort keys %{$hAISCODE_INJURYTYPE{$code}})
#    {
#        print KEYINFO "seq=$hAISCODE_INJURYTYPE{$code}{$injuryType}: [AISCODE,INJURYTYPE]=$code, $injuryType\n";
#    }
#}
#print KEYINFO "\n";
#$runTime = time() - $startTime;
#print LOG "Processed $count two-way [AISCODE,INJURYTYPE] interactions.\n";
#print LOG "Time taken = $runTime seconds.\n\n";

# assign sequence number for [BODYREGION,AISSCORE] combinations
$count = 0;
$startTime = time();
foreach my $key (sort keys %hINC_KEY_BODYREGION_AISSCORE)
{
    %hTMP = %{dclone(\%{$hINC_KEY_BODYREGION_AISSCORE{$key}})};
    foreach my $bodyRegion (sort keys %hTMP)
    {
        foreach my $score (sort keys %{$hTMP{$bodyRegion}})
        {
            if(!exists($hBODYREGION_AISSCORE{$bodyRegion}{$score}))
            {
                $count++;
                print LOG "Processing [BODYREGION,AISSCORE] combinations...$count\n";
                $hBODYREGION_AISSCORE{$bodyRegion}{$score} = 1;
            }
        }
    }
}
foreach my $bodyRegion (sort keys %hBODYREGION_AISSCORE)
{
    foreach my $score (sort keys %{$hBODYREGION_AISSCORE{$bodyRegion}})
    {
        $hBODYREGION_AISSCORE{$bodyRegion}{$score} = $seqNum;
        $seqNum++;
    }
}
print KEYINFO "[BODYREGION,AISSCORE] combination Keys... $count keys\n";
foreach my $bodyRegion (sort keys %hBODYREGION_AISSCORE)
{
    foreach my $score (sort keys %{$hBODYREGION_AISSCORE{$bodyRegion}})
    {
        print KEYINFO "seq=$hBODYREGION_AISSCORE{$bodyRegion}{$score}: [BODYREGION,AISSCORE]=$bodyRegion, $score\n";
    }
}
print KEYINFO "\n";
$runTime = time() - $startTime;
print LOG "Processed $count [BODYREGION,AISSCORE] combinations.\n";
print LOG "Time taken = $runTime seconds.\n\n";

# assign sequence number for [BODYREGION,AISSCORE]-[BODYREGION,AISSCORE] two-way interactions
$count = 0;
$startTime = time();
foreach my $key (sort keys %hINC_KEY_BODYREGION_AISSCORE)
{
    %hTMP = %{dclone(\%{$hINC_KEY_BODYREGION_AISSCORE{$key}})};
    foreach my $bodyRegion1 (sort keys %hTMP)
    {
        foreach my $score1 (sort keys %{$hTMP{$bodyRegion1}})
        {
            delete($hTMP{$bodyRegion1}{$score1});  # to avoid [bodyRegion1,score1]-[bodyRegion1,score1] combinations
            
            foreach my $bodyRegion2 (sort keys %hTMP)
            {
                foreach my $score2 (sort keys %{$hTMP{$bodyRegion2}})
                {
                    if(!exists($hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}{$bodyRegion2}{$score2}))
                    {
                        $count++;
                        print LOG "Processing two-way [BODYREGION,AISSCORE]-[BODYREGION,AISSCORE] interactions...$count\n";
                        $hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}{$bodyRegion2}{$score2} = 1;
                    }
                }
            }
        }
    }
}
foreach my $bodyRegion1 (sort keys %hBODYREGIONAISSCORE_BODYREGIONAISSCORE)
{
    foreach my $score1 (sort keys %{$hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}})
    {
        foreach my $bodyRegion2 (sort keys %{$hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}})
        {
            foreach my $score2 (sort keys %{$hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}{$bodyRegion2}})
            {
                $hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}{$bodyRegion2}{$score2} = $seqNum;
                $seqNum++;
            }
        }
    }
}
print KEYINFO "Two-way [BODYREGION,AISSCORE]-[BODYREGION,AISSCORE] interactions Keys... $count keys\n";
foreach my $bodyRegion1 (sort keys %hBODYREGIONAISSCORE_BODYREGIONAISSCORE)
{
    foreach my $score1 (sort keys %{$hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}})
    {
        foreach my $bodyRegion2 (sort keys %{$hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}})
        {
            foreach my $score2 (sort keys %{$hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}{$bodyRegion2}})
            {
                print KEYINFO "seq=$hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}{$bodyRegion2}{$score2}: [BODYREGION,AISSCORE]-[BODYREGION,AISSCORE]=[$bodyRegion1, $score1], [$bodyRegion2, $score2]\n";
            }
        }
    }
}
print KEYINFO "\n";
$runTime = time() - $startTime;
print LOG "Processed $count two-way [BODYREGION,AISSCORE]-[BODYREGION,AISSCORE] interactions.\n";
print LOG "Time taken = $runTime seconds.\n\n";

# Construct array of valid INC_KEYs
my @inc_key_valid;
foreach my $key (sort keys(%hINC_KEY))
{
    if($hINC_KEY{$key} == 1)
    {
       push(@inc_key_valid,$key);
    }
}
my $nPatients = $#inc_key_valid + 1;

#foreach my $key (@inc_key_valid)
#{
#    print EDSYSBP "$key, $hAGE{$key}, $hEDSYSBP{$key}, $hOUTPUT{$key}, $hLOS{$key}\n";
#}
#
#foreach my $key (@inc_key_valid)
#{
#    print EDRESPRATE "$key, $hAGE{$key}, $hEDRESPRATE{$key}, $hOUTPUT{$key}, $hLOS{$key}\n";
#}
#
#foreach my $key (@inc_key_valid)
#{
#    print EDPULSE "$key, $hAGE{$key}, $hEDPULSE{$key}, $hOUTPUT{$key}, $hLOS{$key}\n";
#}

my ($FILEHD,$FILELD);
# start to print out data 
foreach my $key (@inc_key_valid)
{
    my $index=0;

    if (length($hLOS{$key})==0) {
            print KEYINFO "watch out for $key ...\n";
    }

    if($hYOINJ{$key} == 0)  #Train data
    {
        $FILEHD = $TRAINHD;
        $FILELD = $TRAINLD;
    }
    else                    #Test data
    {
        $FILEHD = $TESTHD;
        $FILELD = $TESTLD;
    }

    print $FILEHD "$hLOS{$key}" . ":" . "$hOUTPUT{$key} ";
    print $FILELD "$hLOS{$key}" . ":" . "$hOUTPUT{$key} ";

    #$index++;
    #print $FILEHD "$index:$hAGE{$key} ";
    #print $FILELD "$index:$hAGE{$key} ";
    #
    #$index++;
    #print $FILEHD "$index:$hGENDER{$key} ";
    #print $FILELD "$index:$hGENDER{$key} ";
    #
    #$index++;
    #print $FILEHD "$index:$hEDSYSBP{$key} ";
    #print $FILELD "$index:$hEDSYSBP{$key} ";
    #
    #$index++;
    #print $FILEHD "$index:$hEDRESPRATE{$key} ";
    #print $FILELD "$index:$hEDRESPRATE{$key} ";
    #
    #$index++;
    #print $FILEHD "$index:$hEDPULSE{$key} ";
    #print $FILELD "$index:$hEDPULSE{$key} ";

    # ICD9 
    foreach my $j (sort keys %{$hINC_KEY_DCODE{$key}})
    {
        print $FILEHD $hDCODE{$j} . ":1 ";
    }
        
    #AISCODE
    foreach my $j (sort keys %{$hINC_KEY_AISCODE{$key}})
    {
        print $FILEHD $hAISCODE{$j} . ":1 ";
    }
    
    ##ECODE
    #foreach my $j (sort keys %{$hINC_KEY_ECODE{$key}})
    #{
    #    print $FILEHD $hECODE{$j} . ":1 ";
    #}
    #
    ##ECODE_ROW
    #foreach my $j (sort keys %{$hINC_KEY_ECODE_ROW{$key}})
    #{
    #    print $FILEHD $hECODE_ROW{$j} . ":1 ";
    #}
    #
    ##ECODE_COL
    #foreach my $j (sort keys %{$hINC_KEY_ECODE_COL{$key}})
    #{
    #    print $FILEHD $hECODE_COL{$j} . ":1 ";
    #}
    #
    ##EDGCSEYE
    #foreach my $j (sort keys %{$hINC_KEY_EDGCSEYE{$key}})
    #{
    #    print $FILEHD $hEDGCSEYE{$j} . ":1 ";
    #}
    #
    ##EDGCSVERB
    #foreach my $j (sort keys %{$hINC_KEY_EDGCSVERB{$key}})
    #{
    #    print $FILEHD $hEDGCSVERB{$j} . ":1 ";
    #}
    #
    ##EDGCSMOTOR
    #foreach my $j (sort keys %{$hINC_KEY_EDGCSMOTOR{$key}})
    #{
    #    print $FILEHD $hEDGCSMOTOR{$j} . ":1 ";
    #}
    #
    ##EDGCSTOTAL
    #foreach my $j (sort keys %{$hINC_KEY_EDGCSTOTAL{$key}})
    #{
    #    print $FILEHD $hEDGCSTOTAL{$j} . ":1 ";
    #    print $FILELD $hEDGCSTOTAL{$j} . ":1 ";
    #}
    #
    ##INJURYTYPE
    #foreach my $j (sort keys %{$hINC_KEY_INJURYTYPE{$key}})
    #{
    #    print $FILEHD $hINJURYTYPE{$j} . ":1 ";
    #    print $FILELD $hINJURYTYPE{$j} . ":1 ";
    #}
    #
    ##INJURYSITE
    #foreach my $j (sort keys %{$hINC_KEY_INJURYSITE{$key}})
    #{
    #    print $FILEHD $hINJURYSITE{$j} . ":1 ";
    #}
    
    # Two-way [ICD9,ICD9] codes
    %hTMP = %{dclone(\%{$hINC_KEY_DCODE{$key}})};
    foreach my $code1(sort keys %{$hINC_KEY_DCODE{$key}})
    {
        delete($hTMP{$code1});		# to avoid [code1,code1] combinations
        foreach my $code2(sort keys %hTMP)
        {
            print $FILEHD $hDCODE_DCODE{$code1}{$code2} . ":1 ";
        }
    }

    ## Two-way [ICD9,INJURYTYPE] codes
    #foreach my $code(sort keys %{$hINC_KEY_DCODE{$key}})
    #{
    #    foreach my $injuryType(sort keys %{$hINC_KEY_INJURYTYPE{$key}})
    #    {
    #        print $FILEHD $hDCODE_INJURYTYPE{$code}{$injuryType} . ":1 ";
    #    }
    #}
    
    # Two-way [AISCODE,AISCODE] codes
    %hTMP = %{dclone(\%{$hINC_KEY_AISCODE{$key}})};
    foreach my $code1(sort keys %{$hINC_KEY_AISCODE{$key}})
    {
        delete($hTMP{$code1});		# to avoid [code1,code1] combinations
        foreach my $code2(sort keys %hTMP)
        {
            print $FILEHD $hAISCODE_AISCODE{$code1}{$code2} . ":1 ";
        }
    }
    
    ## Two-way [AISCODE,INJURYTYPE] codes
    #foreach my $code(sort keys %{$hINC_KEY_AISCODE{$key}})
    #{
    #    foreach my $injuryType(sort keys %{$hINC_KEY_INJURYTYPE{$key}})
    #    {
    #        print $FILEHD $hAISCODE_INJURYTYPE{$code}{$injuryType} . ":1 ";
    #    }
    #}
    
    # [BODYREGION,AISSCORE] combination codes
    foreach my $bodyRegion(sort keys %{$hINC_KEY_BODYREGION{$key}})
    {
        foreach my $score(sort keys %{$hINC_KEY_AISSCORE{$key}})
        {
            #Print only if a particular combination of BODYREGION,AISSCORE exists for a person 
            if(exists($hINC_KEY_BODYREGION_AISSCORE{$key}{$bodyRegion}{$score}))
            {
                print $FILEHD $hBODYREGION_AISSCORE{$bodyRegion}{$score} . ":1 ";
                print $FILELD $hBODYREGION_AISSCORE{$bodyRegion}{$score} . ":1 ";
            }
        }
    }
    
    # Two-way [BODYREGION,AISSCORE]-[BODYREGION,AISSCORE] interaction codes
    %hTMP = %{dclone(\%{$hINC_KEY_BODYREGION_AISSCORE{$key}})};
    foreach my $bodyRegion1 (sort keys %hTMP)
    {
        foreach my $score1 (sort keys %{$hTMP{$bodyRegion1}})
        {
            delete($hTMP{$bodyRegion1}{$score1});  # to avoid [bodyRegion1,score1]-[bodyRegion1,score1] combinations
            foreach my $bodyRegion2 (sort keys %hTMP)
            {
                foreach my $score2 (sort keys %{$hTMP{$bodyRegion2}})
                {
                    if(exists($hINC_KEY_BODYREGION_AISSCORE{$key}{$bodyRegion1}{$score1}) and exists($hINC_KEY_BODYREGION_AISSCORE{$key}{$bodyRegion2}{$score2}))
                    {
                        print $FILEHD $hBODYREGIONAISSCORE_BODYREGIONAISSCORE{$bodyRegion1}{$score1}{$bodyRegion2}{$score2} . ":1 ";
                    }
                }
            }
        }
    }
    
    print $FILEHD "\n";
    print $FILELD "\n";

}

print LOG "Processed $iter lines. Total number of rows = $iter, Total Patients = $nPatients\n";

$ntdb->disconnect();
close(KEYINFO);
close(LOG);
close($TRAINHD);
close($TRAINLD);
close($TESTHD);
close($TESTLD);
close(EDSYSBP);
close(EDRESPRATE);
close(EDPULSE);